package com.chance.cc.crawler.development.scripts.huanqiu;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.*;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2020/12/1 17:27
 * @Description 环球网
 **/
public class HuanqiuCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(HuanqiuCrawlerScript.class);

    public static final String huanqiuEntranceUrl = "https://www.huanqiu.com/";
    public static final String modulesEntranceUrl = "https://\\S*.huanqiu.com";
    public static final String modulesJsonEntranceUrl = "http\\S*://\\S+.huanqiu.com/api/list\\S*\\?node=\\S*&offset=\\d+&limit=20";
    public static final String articleUrl = "https://\\S+.huanqiu.com/article/[a-zA-Z0-9]*";
    public static final String sourceIdUrl = "https://api.comment.huanqiu.com/api/sourceInfo";
    public static final String commentUrl = "https://api.comment.huanqiu.com/api/commentStatistics\\?app_id=fLHTlMN2j3&source_id=\\S*&limit=10\\S*";

    /**
     * 脚本domain定义
     *
     * @return
     */
    @Override
    public String domain() {
        return "huanqiu";
    }

    /**
     * 进入脚本的正则列表
     */
    @Override
    public void initUrlRegulars() {
        addUrlRegular(huanqiuEntranceUrl);
        addUrlRegular(modulesEntranceUrl);
        addUrlRegular(modulesJsonEntranceUrl);
        addUrlRegular(articleUrl);
        addUrlRegular(sourceIdUrl);
        addUrlRegular(commentUrl);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     *
     * @param crawlerRequestRecord
     * @return
     */
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }


    /**
     * 解析链接方法
     *
     * @param crawlerRequestRecord
     * @param httpPage
     * @return
     */
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        Proxy proxy = new Proxy();
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");

        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();

        String requestUrl = httpPage.getRequest().getUrl();
        String webSite = requestUrl.substring(requestUrl.indexOf("//") + 2, requestUrl.indexOf("."));

        if (requestUrl.matches(huanqiuEntranceUrl)) {
            List<String> all = httpPage.getHtml().xpath("//div[@class=\"rTxt\"]/a/@href").all();
            for (String url : all) {
                url = "https:" + url;
                if (url.matches(modulesEntranceUrl)) {
                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .httpUrl(url)
                            .recordKey(url)
                            .releaseTime(System.currentTimeMillis())
                            .notFilterRecord()
                            .proxy(proxy)
                            .build();

                    parsedLinks.add(itemRecord);
                }
            }
        } else if (requestUrl.matches(modulesEntranceUrl)) {
            //查找对应模块的接口

            String jsonUrl = "";
            if (webSite.matches("world")) {
                jsonUrl = "https://world.huanqiu.com/api/list?node=%22/e3pmh22ph/e3pmh2398%22,%22/e3pmh22ph/e3pmh26vv%22,%22/e3pmh22ph/e3pn6efsl%22,%22/e3pmh22ph/efp8fqe21%22&offset=0&limit=20";
            }else if(webSite.matches("china")){
                jsonUrl = "https://china.huanqiu.com/api/list?node=%22/e3pmh1nnq/e3pmh1obd%22,%22/e3pmh1nnq/e3pn61c2g%22,%22/e3pmh1nnq/e3pn6eiep%22,%22/e3pmh1nnq/e3pra70uk%22,%22/e3pmh1nnq/e5anm31jb%22,%22/e3pmh1nnq/e7tl4e309%22&offset=0&limit=20";
            }else if(webSite.matches("mil")){
                jsonUrl = "https://mil.huanqiu.com/api/list?node=%22/e3pmh1dm8/e3pmt7hva%22,%22/e3pmh1dm8/e3pmtdr2r%22,%22/e3pmh1dm8/e3pn62l96%22,%22/e3pmh1dm8/e3pn6f3oh%22&offset=0&limit=20";
            }else if(webSite.matches("taiwan")){
                jsonUrl = "https://taiwan.huanqiu.com/api/list?node=%22/e3pmh1fmg/e3pmh1g6o%22,%22/e3pmh1fmg/e3pmt8gfv%22,%22/e3pmh1fmg/e3pmt8i3n%22,%22/e3pmh1fmg/e3pmt8lic%22,%22/e3pmh1fmg/e3pmunk13%22,%22/e3pmh1fmg/e3pn6elbj%22&offset=0&limit=20";
            }else if(webSite.matches("opinion")){
                jsonUrl = "https://opinion.huanqiu.com/api/list?node=%22/e3pmub6h5/e3pmub75a%22,%22/e3pmub6h5/e3pn00if8%22,%22/e3pmub6h5/e3pn03vit%22,%22/e3pmub6h5/e3pn4bi4t%22,%22/e3pmub6h5/e3pr9baf6%22,%22/e3pmub6h5/e3prafm0g%22,%22/e3pmub6h5/e3prcgifj%22,%22/e3pmub6h5/e81curi71%22,%22/e3pmub6h5/e81cv14rf%22,%22/e3pmub6h5/e81cv14rf/e81cv52ha%22,%22/e3pmub6h5/e81cv14rf/e81cv7hp0%22,%22/e3pmub6h5/e81cv14rf/e81cvaa3q%22,%22/e3pmub6h5/e81cv14rf/e81cvcd7e%22&offset=0&limit=20";
            }else if(webSite.matches("finance")){
                jsonUrl = "https://finance.huanqiu.com/api/list?node=%22/e3pmh1hmp/e3pmh1iab%22,%22/e3pmh1hmp/e3pn46htn%22,%22/e3pmh1hmp/e3pn60gdi%22,%22/e3pmh1hmp/e3pn60gdi/e3pn60h31%22,%22/e3pmh1hmp/e3pn60gdi/e3pru2fi2%22,%22/e3pmh1hmp/e3pn60rs2%22,%22/e3pmh1hmp/e3pn60rs2/e3pn60skq%22,%22/e3pmh1hmp/e3pn60rs2/e3ptlr015%22,%22/e3pmh1hmp/e3pn61831%22,%22/e3pmh1hmp/e3pn61an9%22,%22/e3pmh1hmp/e3pn61chp%22,%22/e3pmh1hmp/e3pn62ihu%22,%22/e3pmh1hmp/e3pn62uuq%22,%22/e3pmh1hmp/e3pn6314j%22,%22/e3pmh1hmp/e3pn6314j/e3pn6323a%22,%22/e3pmh1hmp/e3pn6314j/e3ptma9ah%22,%22/e3pmh1hmp/e3ptkencb%22,%22/e3pmh1hmp/e3ptlrdc9%22,%22/e3pmh1hmp/e3ptlrdc9/e3ptltkc2%22,%22/e3pmh1hmp/e3ptlrdc9/e3ptm2ci2%22,%22/e3pmh1hmp/e7i6qafud%22,%22/e3pmh1hmp/e7i6t8c0j%22,%22/e3pmh1hmp/e7lipkhq1%22,%22/e3pmh1hmp/e7lipkhq1/e7lipkii0%22,%22/e3pmh1hmp/e7lipkhq1/e7o08h1r8%22&offset=0&limit=20";
            }else if(webSite.matches("tech")){
                jsonUrl = "https://tech.huanqiu.com/api/list?node=%22/e3pmh164r/e3pmh2hq8%22,%22/e3pmh164r/e3pmh33i9%22,%22/e3pmh164r/e3pmh356p%22,%22/e3pmh164r/e3pmh3dh4%22,%22/e3pmh164r/e3pmtlao3%22,%22/e3pmh164r/e3pmtm015%22,%22/e3pmh164r/e3pmtnh4j%22,%22/e3pmh164r/e3pn1fd3s%22,%22/e3pmh164r/e3pn46ri0%22,%22/e3pmh164r/e3pn4bn46%22,%22/e3pmh164r/e3pn4gh77%22,%22/e3pmh164r/e3pn4qlss%22,%22/e3pmh164r/e3pn6fo08%22,%22/e3pmh164r/e3ptqlvrg%22&offset=0&limit=20";
            }else if(webSite.matches("auto")){
                jsonUrl = "https://auto.huanqiu.com/api/list?node=%22/e3pmh24qk/e3pmh25cs%22,%22/e3pmh24qk/e3pmtj57c%22,%22/e3pmh24qk/e3pmtkgc2%22,%22/e3pmh24qk/e3pn02mp3%22,%22/e3pmh24qk/e3pn4el6u%22,%22/e3pmh24qk/ej8aajlga%22&offset=0&limit=20";
            }else if(webSite.matches("art")){
                jsonUrl = "https://art.huanqiu.com/api/list?node=%22/e3pokdnam/e3pokdnrj%22&offset=0&limit=20";
            }else if(webSite.matches("go")){
                jsonUrl = "https://go.huanqiu.com/api/list1?node=/e3pmh1tuv/e3pmh1ufv/e3pmh1v35&tags=%22%E5%87%BA%E6%B8%B8%22&offset=0&limit=20";
            }else if(webSite.matches("health")){
                jsonUrl = "https://health.huanqiu.com/api/list?node=%22/e3pmt7dq2/e3pmt7edc%22,%22/e3pmt7dq2/e3pmt904n%22,%22/e3pmt7dq2/e3pmt9htm%22,%22/e3pmt7dq2/e3pmtbedk%22,%22/e3pmt7dq2/e3pmtvmsa%22,%22/e3pmt7dq2/e3pn49kc7%22,%22/e3pmt7dq2/e3pn4cagl%22,%22/e3pmt7dq2/e3pn4f81k%22,%22/e3pmt7dq2/e3pn50ich%22,%22/e3pmt7dq2/e3pn61f01%22,%22/e3pmt7dq2/e3pn6edle%22,%22/e3pmt7dq2/e3pn6gvs7%22,%22/e3pmt7dq2/e3prd5mi2%22,%22/e3pmt7dq2/e3ptds3rp%22,%22/e3pmt7dq2/e3ptds3rp/e3ptds4r8%22,%22/e3pmt7dq2/e3ptt66fi%22&offset=0&limit=20";
            }else if(webSite.matches("qinzi")){
                jsonUrl = "https://qinzi.huanqiu.com/api/list?node=%22/ebggoaf45/ebgmgut3n%22,%22/ebggoaf45/ebgmgut3n/ebgmhivbk%22,%22/ebggoaf45/ebgmi0d9i%22,%22/ebggoaf45/ebgmi0d9i/ebi36o0qb%22,%22/ebggoaf45/ebi37sp8n%22,%22/ebggoaf45/ebi37sp8n/ebi388sjn%22,%22/ebggoaf45/ebi38m5d7%22,%22/ebggoaf45/ebi38m5d7/ebi391j8t%22,%22/ebggoaf45/ebi39m4up%22,%22/ebggoaf45/ebi39m4up/ebi39uvki%22&offset=0&limit=20";
            }else if(webSite.matches("city")){
                jsonUrl = "https://city.huanqiu.com/api/list?node=%22/e3pmh1nv4/e3pn60pg4%22,%22/e3pmh1nv4/e3pn637a1%22,%22/e3pmh1nv4/e3pu0ucu5%22,%22/e3pmh1nv4/e7td1g6u2%22,%22/e3pmh1nv4/e7td1t7it%22,%22/e3pmh1nv4/ecko2h4rc%22,%22/e3pmh1nv4/ecko2on6j%22,%22/e3pmh1nv4/ecko30dgg%22&offset=0&limit=20";
            }else if(webSite.matches("look")){
                jsonUrl = "https://look.huanqiu.com/api/list?node=%22/e3pn4q8sp/e3pn4q9gc%22,%22/e3pn4q8sp/e3pn7ibfm%22,%22/e3pn4q8sp/e3pofukpl%22,%22/e3pn4q8sp/e3pttoksk%22&offset=0&limit=20";
            }else if(webSite.matches("yrd")){
                jsonUrl = "https://yrd.huanqiu.com/api/list?node=%22/e3pu6i31u/eecbiskt4%22,%22/e3pu6i31u/eecc32cqf%22,%22/e3pu6i31u/eecc35cit%22,%22/e3pu6i31u/eecc38g1f%22,%22/e3pu6i31u/eecc3c3fq%22,%22/e3pu6i31u/eecc3f0eh%22,%22/e3pu6i31u/eecc3i4cj%22,%22/e3pu6i31u/eecc3lq5n%22,%22/e3pu6i31u/eecc3om9o%22,%22/e3pu6i31u/eecc3rbnu%22&offset=0&limit=20";
            }else if(webSite.matches("uav")){
                jsonUrl = "https://uav.huanqiu.com/api/list?node=%22/e3pn5tffs/e3pn5tg0p%22,%22/e3pn5tffs/e3pn5tg0p/e3ptjsfnn%22,%22/e3pn5tffs/e3pn6156o%22,%22/e3pn5tffs/e3poit9nf%22,%22/e3pn5tffs/e3poit9nf/e3ptjtt1d%22,%22/e3pn5tffs/e3pshm06b%22,%22/e3pn5tffs/e3pthq9rg%22,%22/e3pn5tffs/e3ptkjts8%22,%22/e3pn5tffs/e5ar1t85q%22,%22/e3pn5tffs/e7rugtglo%22,%22/e3pn5tffs/e7ruia4rh%22,%22/e3pn5tffs/e7siujt5c%22&offset=0&limit=20";
            }

            if (StringUtils.isNotEmpty(jsonUrl)) {
                CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .httpUrl(jsonUrl)
                        .recordKey(jsonUrl)
                        .releaseTime(System.currentTimeMillis())
                        .notFilterRecord()
                        .proxy(proxy)
                        .build();

                parsedLinks.add(itemRecord);
            }
        } else if (requestUrl.matches(modulesJsonEntranceUrl)) {
            //下一页接口地址的拼接
            String[] split = requestUrl.split("offset=");
            String[] split1 = split[1].split("&");
            String nextPageUrl = split[0] + "offset=" + (Integer.parseInt(split1[0]) + 20) + "&" + split1[1];

            if (nextPageUrl.matches(modulesJsonEntranceUrl)) {
                CrawlerRequestRecord moduleRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .releaseTime(System.currentTimeMillis())
                        .needWashed(false)
                        .build();

                parsedLinks.add(moduleRecord);
            }

            //文章列表解析
            List<String> all = httpPage.getJson().jsonPath($_type + ".list").all();
            for (String data : all) {
                JSONObject jsonObject = JSONObject.parseObject(data);
                if (jsonObject.size() > 0) {
                    String aid = jsonObject.getString("aid");
                    String itemUrl = "";
                    if (StringUtils.isEmpty(aid)) {
                        continue;
                    } else {
                        itemUrl = "https://" + webSite + ".huanqiu.com/article/" + aid;
                    }

                    Long releaseTimeToLong = jsonObject.getLong("xtime");
                    if (releaseTimeToLong == null) {
                        continue;
                    }

                    //文章请求
                    CrawlerRequestRecord moduleRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .proxy(proxy)
                            .resultLabelTag(article)
                            .releaseTime(releaseTimeToLong)
                            .build();

                    parsedLinks.add(moduleRecord);

                    //sourceId的获取地址
                    String sourceIdUrl = "https://api.comment.huanqiu.com/api/sourceInfo";
                    CrawlerRequestRecord sourceRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(sourceIdUrl + aid)
                            .httpUrl(sourceIdUrl)
                            .needWashed(false)
                            .releaseTime(releaseTimeToLong)
                            .build();
                    HttpRequest httpRequest = sourceRecord.getHttpRequest();
                    httpRequest.setMethod("POST");
                    Map<String, Object> map = new HashMap<>();
                    map.put("app_id", "fLHTlMN2j3");
                    map.put("article_id", aid);
                    map.put("article_url", itemUrl);
                    map.put("category", webSite + ".huanqiu.com");
                    map.put("title", jsonObject.getString("title"));
                    httpRequest.setRequestBody(HttpRequestBody.form(map, "UTF-8"));

                    httpRequest.addHeader("content-type", "application/x-www-form-urlencoded; charset=UTF-8");

                    Map<String, Object> extras = new HashMap<>();
                    extras.put("aid", aid);
                    extras.put("articleUrl", itemUrl);
                    httpRequest.setExtras(extras);
                    //解决SSL peer shut down incorrectly异常
                    System.setProperty("https.protocols","TLSv1,TLSv1.1,TLSv1.2,SSLv3");

                    parsedLinks.add(sourceRecord);

                }
            }
        } else if (requestUrl.matches(sourceIdUrl)) {
            String code = httpPage.getJson().jsonPath($_type + ".code").get();
            if (code.matches("2000")) {
                String sourceId = httpPage.getJson().jsonPath($_type + ".data.source_id").get();

                if (StringUtils.isNotEmpty(sourceId)) {
                    String commentUrl = "https://api.comment.huanqiu.com/api/commentStatistics?app_id=fLHTlMN2j3&source_id=" + sourceId + "&limit=10";

                    CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(commentUrl)
                            .httpUrl(commentUrl)
                            .resultLabelTag(interaction)
                            .resultLabelTag(comment)
                            .releaseTime(System.currentTimeMillis())
                            .build();

                    commentRecord.getHttpRequest().setExtras(httpPage.getRequest().getExtras());
                    parsedLinks.add(commentRecord);
                }
            } else {
                log.warn(httpPage.getJson().jsonPath($_type + ".msg").get() + "。文章地址是：" + httpPage.getRequest().getExtras().get("articleUrl"));
            }
        } else if (requestUrl.matches(commentUrl)) {
            List<String> all = httpPage.getJson().jsonPath($_type + ".data.comment_data.comments").all();
            if (all.size() > 0) {
                //下一页评论地址解析
                if (all.size() == 10) {
                    JSONObject lastComment = JSONObject.parseObject(all.get(all.size() - 1));
                    String nextPageUrl = requestUrl + "&last_id=" + lastComment.getString("id");
                    String releaseTime = lastComment.getString("created_at");

                    try {
                        long releaseTimeToLong = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm:ss").getTime();
                        CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(nextPageUrl)
                                .httpUrl(nextPageUrl)
                                .resultLabelTag(comment)
                                .releaseTime(releaseTimeToLong)
                                .build();
                        commentRecord.getHttpRequest().setExtras(crawlerRequestRecord.getHttpRequest().getExtras());
                        parsedLinks.add(commentRecord);
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                }
            } else {
                log.info("文章(“" + crawlerRequestRecord.getHttpRequest().getExtras().get("articleUrl") + "”)的评论数为0");
            }
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();

        int statusCode = page.getStatusCode();
        if (statusCode == 200) {
            if (crawlerResultTags.hasDataType(article)) {
                crawlerDataList.add(washArticle(crawlerRecord, page));
            }
            if (crawlerResultTags.hasDataType(interaction)) {
                crawlerDataList.add(washInteraction(crawlerRecord, page));
            }
            if (crawlerResultTags.hasDataType(comment)) {
                List<String> all = page.getJson().jsonPath($_type + ".data.comment_data.comments").all();
                for (String data : all) {
                    crawlerDataList.add(washComment(crawlerRecord, page,data));
                }
            }
        } else {
            log.info("该文章(“" + page.getRequest().getUrl() + "”)下载错误，错误码：" + statusCode);
        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();
        String webSite = itemUrl.substring(itemUrl.indexOf("//") + 2, itemUrl.indexOf("."));

        String title = httpPage.getHtml().xpath("//div[@class=\"t-container-title\"]//text()").get();
        String source = httpPage.getHtml().xpath("//span[@class=\"source\"]/span//text()").get();
        String author = httpPage.getHtml().xpath("//span[@class=\"author\"]/span//text()").get();
        author = StringUtils.isNotEmpty(author) && author.contains("作者：") ? author.substring(author.indexOf("：") + 1) : "";
        String releaseTime = httpPage.getHtml().xpath("//p[@class=\"time\"]/text()").get();

        List<String> articleTextList = httpPage.getHtml().xpath("//section[@data-type=\"rtext\"]//text()").all();
        StringBuffer conents = new StringBuffer();
        for (String articleText : articleTextList) {
            if (StringUtils.isNotEmpty(articleText)) {
                conents.append(StringEscapeUtils.unescapeHtml4(articleText)).append(" ");
            }
        }

        String articleKey = itemUrl.substring(itemUrl.lastIndexOf("/") + 1);

        CrawlerData crawlerData = null;

        crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                .url(itemUrl)
                .releaseTime(crawlerRequestRecord.getReleaseTime())
                .addContentKV(Field_Content, conents.toString().trim())
                .addContentKV(Field_Title, StringEscapeUtils.unescapeHtml4(title).trim())
                .addContentKV(Field_Source, source)
                .addContentKV(Field_Author, StringEscapeUtils.unescapeHtml4(author))
                .build();


        return crawlerData;

    }

    public CrawlerData washInteraction(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();

        String comments = httpPage.getJson().jsonPath($_type + ".data.statistics.comment_sum").get();
        String participation = httpPage.getJson().jsonPath($_type + ".data.statistics.participation_sum").get();

        String articleKey = (String) httpPage.getRequest().getExtras().get("aid");

        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), interaction.enumVal(), articleKey))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("interaction"))
                .url(itemUrl)
                .releaseTime(crawlerRequestRecord.getReleaseTime())
                .addContentKV("participation", participation)//参与人数
                .addContentKV("comments", comments)
                .build();

        return crawlerData;

    }

    public CrawlerData washComment(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage,String data) {
        String itemUrl = httpPage.getRequest().getUrl();

        JSONObject jsonObject = JSONObject.parseObject(data);
        String content = jsonObject.getString("content");
        String likeCount = jsonObject.getString("like_count");
        String loc = jsonObject.getString("loc");
        String replyCount = jsonObject.getString("reply_count");
        String releaseTime = jsonObject.getString("created_at");
        String id = jsonObject.getString("id");
        JSONObject userInfo = jsonObject.getJSONObject("user_info");
        String avatar = userInfo.getString("avatar");
        String uid = userInfo.getString("uid");
        String username = userInfo.getString("username");



        String articleKey = (String) httpPage.getRequest().getExtras().get("aid");

        CrawlerData crawlerData = null;
        try {
            crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), comment.enumVal(), id))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("comment"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm:ss").getTime())
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                    .addContentKV("content", content)
                    .addContentKV("location",loc)
                    .addContentKV("author", username)
                    .build();
        } catch (ParseException e) {
            e.printStackTrace();
        }


        return crawlerData;

    }

    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }
}
